<?php

class ScrapyScript extends ScriptBase {
	private $scrapyCollection = null;
	private $weixinMpListFinaCollection = null;
	private $mongo = null;
    private $api = null;
    private $common = null;
    private $search = null;
    private $replace = null;
    private $weixinFinaArticleCollection = null;
    private $sogouCookieArr = null;

	public function __construct()
	{
		$this->sogouCookieArr = [];
		$this->mongo = new MongoClient("mongodb://10.16.43.46:27017");
		$this->scrapyCollection = $this->mongo->selectDB('scrapy')->selectCollection('scrapy');
		$this->weixinMpListFinaCollection = $this->mongo->selectDB('weixin')->selectCollection('mpListFina');
		$this->weixinFinaArticleCollection = $this->mongo->selectDB('weixin')->selectCollection('weixinFinaArticle');
		$this->api = n_api('scrapy');
        $this->common = n_api('common');
        $this->search = [
			"'<script[^>]*?>.*?</script>'si", // 去掉 javascript 
			"'<style[^>]*?>.*?</style>'si", // 去掉 css
			"'//<!--[/!]*?[^<>]*?>'si", // 去掉 注释标记
			"'<!--[/!]*?[^<>]*?>'si", // 去掉 注释标记
		];
		$this->replace = [
			"", 
			"",
			"",
			""
		];
	}

	private function postSogou($url, $sogouid)
	{
		sleep(2);

		if (!array_key_exists($sogouid, $this->sogouCookieArr)) {
			$this->sogouCookieArr[$sogouid] = $this->api->getTaskCookies('75a2a6f643c76391ad5b0c7535e467bf')[0];
		}
		$cookieStr = '';
		$tmp = json_decode($this->sogouCookieArr[$sogouid], true);
		foreach ($tmp as $value) {
			$cookieStr .= $value['Name'] . '=' . $value['Value'] . '; ';
		}
		$cookieStr = rtrim($cookieStr, ';');
		$ch = curl_init($url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.36');
		curl_setopt($ch, CURLOPT_HTTPHEADER, [
			'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
			'Accept-Language: zh-CN,zh;q=0.8',
			'Connection: keep-alive',
			'Host: weixin.sogou.com',
			'Referer: http://weixin.sogou.com/'
		]);
		curl_setopt($ch, CURLOPT_COOKIE, $cookieStr);
		
		$respond = curl_exec($ch);
		$curlRs = curl_getinfo($ch);		
		curl_close($ch);
		$rs['header'] = $curlRs;
		$rs['respond'] = $respond;

		return $rs;
	}

	private function clearSogouCookie($sogouid, $isDel)
	{
		if ($isDel == 'N') {
			echo $this->sogouCookieArr[$sogouid] . " return to pool\n";
			$this->api->addCookie('75a2a6f643c76391ad5b0c7535e467bf', $this->sogouCookieArr[$sogouid]);	
		} else {
			echo $this->sogouCookieArr[$sogouid] . " deleted\n";
		}
		unset($this->sogouCookieArr[$sogouid]);
	}

	private function handleSogouXml($str, $sogouid, $wxid)
	{
		$isDel = 'N';
		$weixin = n_api('weixin');
		$fixContent = trim(substr(trim(preg_replace($this->search, $this->replace, $str)), 19), ')');
		$arr = json_decode($fixContent, true);
		if ($arr['totalItems'] == 0) {
			echo "no articles\n";
		}else{
			$items = $arr['items'];
			foreach ($items as $v) {
				$xmlItem = simplexml_load_string(str_replace('gbk', 'utf-8', $v));
				$docId = $xmlItem->item->display->docid;
				$Exists = $this->weixinFinaArticleCollection->findOne(['sogouDocId' => $docId]);
				if ($Exists == NULL) {
					$articleMiddleUrl = 'http://weixin.sogou.com' . trim($xmlItem->item->display->url);
					
					$rs = $this->postSogou($articleMiddleUrl, $sogouid);
					$content = $rs['respond'];
					if (strpos($content, 'The URL has moved') !== false) {
						$tmp = substr($content, 27);
						$url = substr($tmp, 0, strlen($tmp) - 10);
						echo 'the url is: ' . $url . "\n";
						$content = file_get_contents($url);
					}
					preg_match("/var\sbiz.*\n/ui", $content, $bizArr);
					$biz = trim(trim(substr($bizArr[0], 11)), '";');
					if (!$biz) {
						echo "biz is false\n";
						var_dump($content);
						$isDel = 'Y';
						continue;
					}
					preg_match("/var\ssn.*\n/ui", $content, $snArr);
					$tmp = trim(substr($snArr[0], 10));
					$sn = substr($tmp, 0, strlen($tmp) - 8);
					preg_match("/var\smid.*\n/ui", $content, $midArr);
					$tmp = trim(substr($midArr[0], 11));
					$mid = substr($tmp, 0, strlen($tmp) - 8);
					preg_match("/var\sidx.*\n/ui", $content, $idxArr);
					$tmp = trim(substr($idxArr[0], 11));
					$idx = substr($tmp, 0, strlen($tmp) - 9);
					$articleUrl = $weixin->generateArticleUrl($biz, $mid, $idx, $sn);
					$query = ['biz' => $biz, 'mid' => $mid];
					$tmpRs = $this->weixinFinaArticleCollection->findOne($query);
					$data = [
						'sogouId' => $sogouid,
						'sogouDocId' => $docId,
						'wxId' => $wxid,
						'biz' => $biz,
						'mid' => $mid,
						'idx' => $idx,
						'sn' => $sn,
						'articleUrl' => $articleUrl,
						'content' => $content,
						'ctime' => time()
					];
					$this->weixinFinaArticleCollection->insert($data);
					echo $articleUrl . " added successfully!\n";
					unset($rs, $content);
				} else {
					echo $docId . " added already!\n";
				}
				unset($xmlItem, $Exists);
			} //end foreach
		} // end else
		$this->clearSogouCookie($sogouid, $isDel);
	}

	public function handleSogou()
	{
		set_time_limit(0);
		ini_set('memory_limit', '3072M');

		$sogou = n_api('sogou');
		$taskId = '75a2a6f643c76391ad5b0c7535e467bf';

		while (true) {
			$query = ['flag' => 2, 'sogouid' => ['$ne' => null]];
			$cursor = $this->weixinMpListFinaCollection->find($query);
			$allWxids = iterator_to_array($cursor);
			foreach ($allWxids as $value) {
				$url = $sogou->getSogouUrl($value['sogouid']);
				$rs = $this->postSogou($url, $value['sogouid']);
				$this->handleSogouXml($rs['respond'], $value['sogouid'], $value['wxid']);
			}
			echo "one round finished\n";
			sleep(60);
		}
	}
}